"""
爬虫大作业： 智联招聘数据爬取
"""

import scrapy
from scrapy import Request, Selector

from sipderWorkScrapy.items import JobInfo


class GetjobsinfoSpider(scrapy.Spider):
    name = 'getjobsinfo'
    allowed_domains = ['zhaopin.com']

    # start_urls = ['https://www.zhaopin.com/']
    def start_requests(self):
        yield Request(url='https://www.zhaopin.com/')

    def parse(self, response, **kwargs):
        jobs_list = Selector(text=response.text).xpath(
            "//a[@class='zp-jobNavigater__pop--href']/text()").extract()  # 工作列表
        citys_list = Selector(text=response.text).xpath(
            "//div[@class='footerFuncCity clearfix']/ul/li/strong/a/text()").extract()  # 工作地点
        print(jobs_list)
        print(citys_list)

        for job in jobs_list:
            for city in citys_list:
                url = f'http://sou.zhaopin.com/?jl={city}&kw={job}'
                yield Request(url=url, callback=self.jobs_parse)

    def jobs_parse(self, response):
        doms = Selector(text=response.text).xpath(
            "//*[@id='positionList-hook']/div/div[@class='joblist-box__item clearfix']").extract()
        for dom in doms:
            job_name = Selector(text=dom).xpath(
                "//span[@class='iteminfo__line1__jobname__name']/@title").extract_first()  # 岗位名称
            job_salary = Selector(text=dom).xpath(
                "//p[@class='iteminfo__line2__jobdesc__salary']/text()").extract_first()  # 薪资范围
            job_place = Selector(text=dom).xpath(

                "//ul[@class='iteminfo__line2__jobdesc__demand']/li[1]/text()").extract_first()  # 工作地点
            job_experience = Selector(text=dom).xpath(
                "//ul[@class='iteminfo__line2__jobdesc__demand']/li[2]/text()").extract_first()  # 经验要求
            job_education = Selector(text=dom).xpath(
                "//ul[@class='iteminfo__line2__jobdesc__demand']/li[3]/text()").extract_first()  # 学历要求
            job_tag = Selector(text=dom).xpath(
                "//div[@class='iteminfo__line3__welfare']/div/text()").extract()  # 岗位标签
            company_name = Selector(text=dom).xpath(
                "//span[@class='iteminfo__line1__compname__name']/@title").extract_first()  # 公司名称
            company_type = Selector(text=dom).xpath(
                "//div[@class='iteminfo__line2__compdesc']/span[1]/text()").extract_first()  # 企业类型
            company_scale = Selector(text=dom).xpath(
                "//div[@class='iteminfo__line2__compdesc']/span[2]/text()").extract_first()  # 企业规模
            link = Selector(text=dom).xpath(
                "//a[@class='joblist-box__iteminfo iteminfo']/@href").extract_first()  # 岗位链接
            job_info = JobInfo()
            job_info['job_name'] = job_name
            job_info['job_salary'] = job_salary
            job_info['job_place'] = job_place
            job_info['job_experience'] = job_experience
            job_info['job_education'] = job_education
            job_info['job_tag'] = job_tag
            job_info['company_name'] = company_name
            job_info['company_type'] = company_type
            job_info['company_scale'] = company_scale
            job_info['link'] = link
            # print(job_info)
            yield job_info

